'''
Created on 2011-11-23

@author: Administrator
'''
from foxtrader import *
import datetime
from processbydate import *
def date2int(sdate):
    return sdate.year*10000+sdate.month*100+sdate.day
def str2date(sdate):
    return sdate
    return datetime.date(int(sdate[:4]),int(sdate[4:6]),int(sdate[6:8]))
def int2date(s):
    sdate=str(s)
    if sdate.find('/')<0 and sdate.find('-')<0 :
        return datetime.date(int(sdate[:4]),int(sdate[4:6]),int(sdate[6:8]))
    sdate=sdate.replace('/','-')
    s=[]
    s=sdate.split('-')
    return datetime.date(int(s[0]),int(s[1]),int(s[2]))
class processByDate():
    def __init__(self):
        if os.path.exists(stockprocessbydatepath)==False: os.mkdir(stockprocessbydatepath)
        for reg in reglist2:
            regpath=os.path.join(stockprocessbydatepath,reg)
            if os.path.exists(regpath)==False: os.mkdir(regpath)
            for d in ['day','week','month']:
                dpath=os.path.join(stockprocessbydatepath,reg,d)
                if os.path.exists(dpath)==False: os.mkdir(dpath)
        
        pass
    def getfilelist(self):
        stocklist={}
        for root,dirs,files in os.walk(rawcontentpath):
            for f in files:
#                print f
#                print os.path.join(root,f)
                if f.find('.'): stockcode=f[:-4]
                else: stockcode=f
                stocklist[stockcode]=os.path.join(root,f)
#        print stocklist
        return stocklist
    def processonestock(self,stockcode,stockfilename):
        def loadstockfromfile(stockfilename):
            clist=open(stockfilename,'r').read().split('\n')
            tlist=[]
            for t in range(1,len(clist)-2):
                r={}
                klist=clist[t].split('"')
                r['date']=int(klist[0])
                r['open']=myfloat(klist[1])
                r['high']=myfloat(klist[2])
                r['low']=myfloat(klist[3])
                r['close']=myfloat(klist[4])
                r['vol']=myfloat(klist[5])
                tlist.append(r)
            return tlist        
        stockdata=loadstockfromfile(stockfilename)
        for daydata in stockdata:
            self.dp.add(daydata)
            self.wp.add(daydata)
            self.mp.add(daydata)
        dret=self.dp.getresult()
        wret=self.wp.getresult()
        mret=self.mp.getresult()        
        
        for d in dret:
            reg=stockcode[:2]
#            print reg,d
            fname=os.path.join(stockprocessbydatepath,reg,'day',str(d['date']))
            if os.path.exists(fname)==False: 
                f=open(fname,'w')
                f.close()
            f=open(fname,'a')
            d['code']=stockcode
            f.write(json.dumps(d))
            f.write('\n')
            f.close
        for d in wret:
            reg=stockcode[:2]
#            print reg,d
            fname=os.path.join(stockprocessbydatepath,reg,'week',str(d['date']))
            if os.path.exists(fname)==False: 
                f=open(fname,'w')
                f.close()
            f=open(fname,'a')
            d['code']=stockcode
            f.write(json.dumps(d))
            f.write('\n')
            f.close        
        for d in mret:
            reg=stockcode[:2]
#            print reg,d
            fname=os.path.join(stockprocessbydatepath,reg,'month',str(d['date']/100))
            if os.path.exists(fname)==False: 
                f=open(fname,'w')
                f.close()
            f=open(fname,'a')
            d['code']=stockcode
            f.write(json.dumps(d))
            f.write('\n')
            f.close                        
        
        
        print stockcode,len(stockdata),len(dret),len(wret),len(mret)
        pass
    def process(self,reg):
        def loadstockfromfile(stockfilename):
            clist=open(stockfilename,'r').read().split('\n')
            tlist=[]
            for t in range(1,len(clist)-2):
                r={}
                klist=clist[t].split('"')
                r['date']=int(klist[0])
                r['open']=myfloat(klist[1])
                r['high']=myfloat(klist[2])
                r['low']=myfloat(klist[3])
                r['close']=myfloat(klist[4])
                r['vol']=myfloat(klist[5])
                tlist.append(r)
            return tlist   
        fd=FoxTrader()
        stocklist=fd.readdayfile(reg)
        print starttime,(datetime.datetime.now()-starttime).seconds
#        stocklist=self.getfilelist()
        daylist={}
        weeklist={}
        monthlist={}
        i=0
        for code in stocklist.keys():
            dp=dayProcess()
            wp=weekProcess()
            mp=monthProcess()
            i=i+1
#            print code,stocklist[code],i,starttime,(datetime.datetime.now()-starttime).seconds
            datalist=stocklist[code]
            while len(datalist)>0:
                oneday=datalist.pop(0)
                dp.add(oneday)
                wp.add(oneday)
                mp.add(oneday) 
            dret=dp.getresult()
            dp=None
            wret=wp.getresult()
            wp=None
            mret=mp.getresult()  
            mp=None
            for d in dret:
                d['code']=code
                if d['date'] in daylist.keys():
                    daylist[d['date']].append(d)
                else:
                    daylist[d['date']]=[]
                    daylist[d['date']].append(d)   
            dret=None
            for d in wret:
                d['code']=code
                if d['date'] in weeklist.keys():
                    weeklist[d['date']].append(d)
                else:
                    weeklist[d['date']]=[]
                    weeklist[d['date']].append(d) 
            wret=None 
            for d in mret:
                d['code']=code
                if d['date'] in monthlist.keys():
                    monthlist[d['date']].append(d)
                else:
                    monthlist[d['date']]=[]
                    monthlist[d['date']].append(d) 
            mret=None
        print starttime,(datetime.datetime.now()-starttime).seconds
######################################################################  
        self.removelastfile(reg,'day')
        self.removelastfile(reg,'week')
        self.removelastfile(reg,'month')

        self.dump2file(reg, 'day', daylist)
        self.dump2file(reg, 'week', weeklist)
        self.dump2file(reg, 'month', monthlist)
#        for d in weeklist.keys():
#            fname=os.path.join(stockprocessbydatepath,reg,'week',str(d))
#            if os.path.exists(fname)==False:
#                f=open(fname,'w')            
#                f.write(json.dumps(weeklist[d]))
#                f.close  
#        for d in monthlist.keys():
#            fname=os.path.join(stockprocessbydatepath,reg,'month',str(d))
#            if os.path.exists(fname)==False:
#                f=open(fname,'w')            
#                f.write(json.dumps(monthlist[d]))
#                f.close    
    def removelastfile(self,reg,duration):        
        dlist=[]
        dpath=os.path.join(stockprocessbydatepath,reg,duration)
#        print dpath
        for root,dirs,files in os.walk(dpath):
            for f in files:
                dlist.append({'code':f,'filename':os.path.join(root,f)})
        if len(dlist)>0:
            lastone=dlist.pop()
    #        print 'last ',duration,lastone['filename']     
            os.remove(lastone['filename'])
    def dump2file(self,reg,duration,datalist):
        for d in datalist.keys():
            fname=os.path.join(stockprocessbydatepath,reg,duration,str(d))
            if os.path.exists(fname)==False:
                f=open(fname,'w')            
                f.write(json.dumps(datalist[d]))
                f.close           
        pass
if __name__ == '__main__':
#    fd=FoxTrader()
#    stockdata=fd.readdayfile('HK')
    starttime=datetime.datetime.now()
    print starttime   
#    downloadandprocessonestock()
    pbd=processByDate()
    for reg in reglist2:
        pbd.process(reg)
#    out=outputResult()
#    out.outputM120('HK')
    print 'download OK',starttime,(datetime.datetime.now()-starttime).seconds
    pass